{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import io\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim\n",
    "from six.moves import cPickle as pickle\n",
    "import numpy as np\n",
    "import scipy.stats as stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "truthful_pos = 'op_spam_v1.4/positive_polarity/truthful_from_TripAdvisor/'\n",
    "truthful_neg = 'op_spam_v1.4/negative_polarity/truthful_from_Web/'\n",
    "\n",
    "deceptive_pos = 'op_spam_v1.4/positive_polarity/deceptive_from_MTurk/'\n",
    "deceptive_neg = 'op_spam_v1.4/negative_polarity/deceptive_from_MTurk/'\n",
    "\n",
    "truthful_reviews_link = []\n",
    "\n",
    "for fold in os.listdir(truthful_pos):\n",
    "    foldLink = os.path.join(truthful_pos, fold)\n",
    "    if os.path.isdir(foldLink):\n",
    "        for f in os.listdir(foldLink):\n",
    "            fileLink = os.path.join(foldLink, f)\n",
    "            truthful_reviews_link.append(fileLink)\n",
    "\n",
    "for fold in os.listdir(truthful_neg):\n",
    "    foldLink = os.path.join(truthful_neg, fold)\n",
    "    if os.path.isdir(foldLink):\n",
    "        for f in os.listdir(foldLink):\n",
    "            fileLink = os.path.join(foldLink, f)\n",
    "            truthful_reviews_link.append(fileLink)\n",
    "\n",
    "deceptive_reviews_link = []\n",
    "\n",
    "for fold in os.listdir(deceptive_pos):\n",
    "    foldLink = os.path.join(deceptive_pos, fold)\n",
    "    if os.path.isdir(foldLink):\n",
    "        for f in os.listdir(foldLink):\n",
    "            fileLink = os.path.join(foldLink, f)\n",
    "            deceptive_reviews_link.append(fileLink)\n",
    "\n",
    "for fold in os.listdir(deceptive_neg):\n",
    "    foldLink = os.path.join(deceptive_neg, fold)\n",
    "    if os.path.isdir(foldLink):\n",
    "        for f in os.listdir(foldLink):\n",
    "            fileLink = os.path.join(foldLink, f)\n",
    "            deceptive_reviews_link.append(fileLink)\n",
    "        \n",
    "print('Number of truthfuls reviews ', len(truthful_reviews_link))\n",
    "print('Number of deceptives reviews ', len(deceptive_reviews_link))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for all datasets except for SST.\n",
    "    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"[^A-Za-z0-9(),!?\\'\\`]\", \" \", string)\n",
    "    string = re.sub(r\"\\'s\", \" \\'s\", string)\n",
    "    string = re.sub(r\"\\'ve\", \" \\'ve\", string)\n",
    "    string = re.sub(r\"n\\'t\", \" n\\'t\", string)\n",
    "    string = re.sub(r\"\\'re\", \" \\'re\", string)\n",
    "    string = re.sub(r\"\\'d\", \" \\'d\", string)\n",
    "    string = re.sub(r\"\\'ll\", \" \\'ll\", string)\n",
    "    string = re.sub(r\",\", \" , \", string)\n",
    "    string = re.sub(r\"!\", \" ! \", string)\n",
    "    string = re.sub(r\"\\(\", \" \\( \", string)\n",
    "    string = re.sub(r\"\\)\", \" \\) \", string)\n",
    "    string = re.sub(r\"\\?\", \" \\? \", string)\n",
    "    string = re.sub(r\"\\s{2,}\", \" \", string)\n",
    "    return string.strip().lower()\n",
    "\n",
    "def handleFile(filePath):\n",
    "    with open(filePath, \"r\") as f:\n",
    "        lines=f.readlines()\n",
    "        file_voc = []\n",
    "        file_numWords = 0\n",
    "        for line in lines:\n",
    "            cleanedLine = clean_str(line)\n",
    "            cleanedLine = cleanedLine.strip()\n",
    "            cleanedLine = cleanedLine.lower()\n",
    "            words = cleanedLine.split(' ')\n",
    "            file_numWords = file_numWords + len(words)\n",
    "            file_voc.extend(words)\n",
    "    return file_voc, file_numWords\n",
    "\n",
    "\n",
    "allFilesLinks = truthful_reviews_link + deceptive_reviews_link\n",
    "vocabulary = []\n",
    "numWords = []\n",
    "for fileLink in allFilesLinks:\n",
    "    file_voc, file_numWords = handleFile(fileLink)\n",
    "    vocabulary.extend(file_voc)\n",
    "    numWords.append(file_numWords)\n",
    "\n",
    "vocabulary = set(vocabulary)\n",
    "vocabulary = list(vocabulary)\n",
    "\n",
    "print('The total number of files is ', len(numWords))\n",
    "print('The total number of words in the files is ', sum(numWords))\n",
    "print('Vocabulary size is ', len(vocabulary))\n",
    "print('The average number of words in the files is', sum(numWords)/len(numWords))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"Visualize the data in histogram format\"\"\"\n",
    "plt.hist(numWords, 50)\n",
    "plt.xlabel('Sequence Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2v_model = gensim.models.Word2Vec.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)\n",
    "wordsVectors = []\n",
    "notFoundwords = []\n",
    "for word in vocabulary:\n",
    "    try:\n",
    "        vector = w2v_model[word]\n",
    "        wordsVectors.append(vector)\n",
    "    except Exception as e:\n",
    "        notFoundwords.append(word)\n",
    "        wordsVectors.append(np.random.uniform(-0.25,0.25,300))  \n",
    "\n",
    "del w2v_model\n",
    "wordsVectors = np.asarray(wordsVectors)\n",
    "\n",
    "print('The number of missing words is ', len(notFoundwords))\n",
    "\n",
    "\"\"\"Save\"\"\"\n",
    "pickle_file = os.path.join('/Users/MacBook/Documents/MLTraining/DECEPTIVE_REVIEWS_ON_HOTEL/', 'save.pickle')\n",
    "\n",
    "try:\n",
    "    f = open(pickle_file, 'wb')\n",
    "    save = {\n",
    "        'wordsVectors': wordsVectors,\n",
    "        'vocabulary': vocabulary,\n",
    "        'notFoundwords': notFoundwords\n",
    "    }\n",
    "    \n",
    "    pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
    "    f.close()\n",
    "except Exception as e:\n",
    "    print('Unable to save data to', pickle_file, ':', e)\n",
    "    raise\n",
    "\n",
    "statinfo = os.stat(pickle_file)\n",
    "print('Compressed pickle size:', statinfo.st_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQ_LENGTH = 160\n",
    "def convertFileToIndexArray(filePath):\n",
    "    doc = np.zeros(MAX_SEQ_LENGTH, dtype='int32')\n",
    "    with open(filePath, \"r\") as f:\n",
    "        lines=f.readlines()\n",
    "        indexCounter = 0\n",
    "        for line in lines:\n",
    "            cleanedLine = clean_str(line)\n",
    "            cleanedLine = cleanedLine.strip()\n",
    "            cleanedLine = cleanedLine.lower()\n",
    "            words = cleanedLine.split(' ')\n",
    "            for word in words:\n",
    "                doc[indexCounter] = vocabulary.index(word)\n",
    "                indexCounter = indexCounter + 1\n",
    "                if (indexCounter >= MAX_SEQ_LENGTH):\n",
    "                    break\n",
    "            if (indexCounter >= MAX_SEQ_LENGTH):\n",
    "                break\n",
    "    return doc\n",
    "\n",
    "totalFiles = len(truthful_reviews_link) + len(deceptive_reviews_link)\n",
    "idsMatrix = np.ndarray(shape=(totalFiles, MAX_SEQ_LENGTH), dtype='int32')\n",
    "labels = np.ndarray(shape=(totalFiles, 2), dtype='int32')\n",
    "\n",
    "counter = 0\n",
    "for filePath in truthful_reviews_link:\n",
    "    idsMatrix[counter] = convertFileToIndexArray(filePath)\n",
    "    counter = counter + 1\n",
    "\n",
    "for filePath in deceptive_reviews_link:\n",
    "    idsMatrix[counter] = convertFileToIndexArray(filePath)\n",
    "    counter = counter + 1\n",
    "    \n",
    "labels[0:len(truthful_reviews_link)] = np.array([1, 0])\n",
    "labels[len(truthful_reviews_link):totalFiles] = np.array([0, 1])\n",
    "\n",
    "print('The shape of the ids matrix is ', idsMatrix.shape)\n",
    "print('The shape of the labels is ', labels.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Create a training set, a validation set and a test set after mixing the data\n",
    "80% for the training set\n",
    "10% for the validation set\n",
    "10% for the test set\n",
    "\"\"\"\n",
    "size = idsMatrix.shape[0]\n",
    "testSize = int(size * 0.1)\n",
    "shuffledIndex = np.random.permutation(size)\n",
    "testIndexes = shuffledIndex[0:testSize]\n",
    "validationIndexes = shuffledIndex[testSize:2*testSize]\n",
    "trainIndexes = shuffledIndex[2*testSize:size]\n",
    "\n",
    "test_data = idsMatrix[testIndexes]\n",
    "test_labels = labels[testIndexes]\n",
    "\n",
    "validation_data = idsMatrix[validationIndexes]\n",
    "validation_labels = labels[validationIndexes]\n",
    "\n",
    "train_data = idsMatrix[trainIndexes]\n",
    "train_labels = labels[trainIndexes]\n",
    "\n",
    "print('train data shape ', train_data.shape)\n",
    "print('train labels shape ', train_labels.shape)\n",
    "print('validation data shape ', validation_data.shape)\n",
    "print('validation labels shape ', validation_labels.shape)\n",
    "print('test data shape ', test_data.shape)\n",
    "print('test labels shape ', test_labels.shape)\n",
    "\n",
    "pickle_file = os.path.join('/Users/MacBook/Documents/MLTraining/DECEPTIVE_REVIEWS_ON_HOTEL/', 'data_saved.pickle')\n",
    "\n",
    "try:\n",
    "    f = open(pickle_file, 'wb')\n",
    "    save = {\n",
    "        'train_data': train_data,\n",
    "        'train_labels': train_labels,\n",
    "        'validation_data': validation_data,\n",
    "        'validation_labels': validation_labels,\n",
    "        'test_data': test_data,\n",
    "        'test_labels': test_labels\n",
    "    }\n",
    "    \n",
    "    pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
    "    f.close()\n",
    "except Exception as e:\n",
    "    print('Unable to save data to', pickle_file, ':', e)\n",
    "    raise\n",
    "\n",
    "statinfo = os.stat(pickle_file)\n",
    "print('Compressed pickle size:', statinfo.st_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
